// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#ifndef __ST_PPL_KERNEL_RISCV_FP32_FC_VEC128_KERNEL_FC_NDARRAY_KERNEL_FP32_VEC128_H_
#define __ST_PPL_KERNEL_RISCV_FP32_FC_VEC128_KERNEL_FC_NDARRAY_KERNEL_FP32_VEC128_H_

#include "ppl/kernel/riscv/common/fc/fc_ndarray_common.h"
#include "ppl/kernel/riscv/common/internal_include.h"

namespace ppl { namespace kernel { namespace riscv {

template<int64_t m, int64_t n, bool first>
inline void fc_ndarray_gemm_kernel_m7n16_fp32_vec128(const float* a, const float* b, const float* bias, float* c, const int32_t total_m, const int32_t total_n, const int32_t total_k) {
    const int64_t k_blk = 4;
    
    asm volatile(
        ".equ            IS_FIRST, %c[IS_FIRST]                     \n\t"
        ".equ            ATOM_M, %c[ATOM_M]                         \n\t"
        ".equ            ATOM_N, %c[ATOM_N]                         \n\t"

        "addi            s3, zero, 4                                \n\t"
        "vsetvli         s2, s3, e32                                \n\t"

        "mv              s2, %[A_LOC]                               \n\t"
        "mv              s3, %[B_LOC]                               \n\t"
        "mv              s4, %[C_LOC]                               \n\t"
        "mv              s5, %[K]                                   \n\t"
        "mv              s6, %[BIAS_LOC]                            \n\t"

    "0:                                                             \n\t" // load C
        ".if IS_FIRST == 1                                          \n\t"
        ".if ATOM_N > 0                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vle.v           v4, (s6)                                   \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vmv.v.v         v8, v4                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vmv.v.v         v12, v4                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vmv.v.v         v16, v4                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vmv.v.v         v20, v4                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vmv.v.v         v24, v4                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vmv.v.v         v28, v4                                    \n\t"
        ".endif                                                     \n\t"
        "add             s6, s6, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vle.v           v5, (s6)                                   \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vmv.v.v         v9, v5                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vmv.v.v         v13, v5                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vmv.v.v         v17, v5                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vmv.v.v         v21, v5                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vmv.v.v         v25, v5                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vmv.v.v         v29, v5                                    \n\t"
        ".endif                                                     \n\t"
        "add             s6, s6, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vle.v           v6, (s6)                                   \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vmv.v.v         v10, v6                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vmv.v.v         v14, v6                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vmv.v.v         v18, v6                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vmv.v.v         v22, v6                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vmv.v.v         v26, v6                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vmv.v.v         v30, v6                                    \n\t"
        ".endif                                                     \n\t"
        "add             s6, s6, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vle.v           v7, (s6)                                   \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vmv.v.v         v11, v7                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vmv.v.v         v15, v7                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vmv.v.v         v19, v7                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vmv.v.v         v23, v7                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vmv.v.v         v27, v7                                    \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vmv.v.v         v31, v7                                    \n\t"
        ".endif                                                     \n\t"
        "add             s6, s6, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".else                                                      \n\t"
        ".if ATOM_M > 0                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v4, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v5, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v6, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v7, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v8, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v9, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v10, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v11, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v12, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v13, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v14, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v15, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v16, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v17, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v18, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v19, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v20, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v21, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v22, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v23, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v24, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v25, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v26, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v27, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v28, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v29, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v30, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v31, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        "mv              s4, %[C_LOC]                               \n\t"
        ".endif                                                     \n\t"

    "1:                                                             \n\t" // loop k
        "addi            s5, s5, -4                                 \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "flw             f0, 0(s2)                                  \n\t"
        "flw             f1, 4(s2)                                  \n\t"
        "flw             f2, 8(s2)                                  \n\t"
        "flw             f3, 12(s2)                                 \n\t"
        ".if ATOM_M <= 1                                            \n\t"
        "add             s2, s2, %[A_BACK_STRIDE]                   \n\t"
        ".else                                                      \n\t"
        "add             s2, s2, %[A_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "flw             f4, 0(s2)                                  \n\t"
        "flw             f5, 4(s2)                                  \n\t"
        "flw             f6, 8(s2)                                  \n\t"
        "flw             f7, 12(s2)                                 \n\t"
        ".if ATOM_M <= 2                                            \n\t"
        "add             s2, s2, %[A_BACK_STRIDE]                   \n\t"
        ".else                                                      \n\t"
        "add             s2, s2, %[A_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "flw             f8, 0(s2)                                  \n\t"
        "flw             f9, 4(s2)                                  \n\t"
        "flw             f10, 8(s2)                                 \n\t"
        "flw             f11, 12(s2)                                \n\t"
        ".if ATOM_M <= 3                                            \n\t"
        "add             s2, s2, %[A_BACK_STRIDE]                   \n\t"
        ".else                                                      \n\t"
        "add             s2, s2, %[A_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "flw             f12, 0(s2)                                 \n\t"
        "flw             f13, 4(s2)                                 \n\t"
        "flw             f14, 8(s2)                                 \n\t"
        "flw             f15, 12(s2)                                \n\t"
        ".if ATOM_M <= 4                                            \n\t"
        "add             s2, s2, %[A_BACK_STRIDE]                   \n\t"
        ".else                                                      \n\t"
        "add             s2, s2, %[A_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "flw             f16, 0(s2)                                 \n\t"
        "flw             f17, 4(s2)                                 \n\t"
        "flw             f18, 8(s2)                                 \n\t"
        "flw             f19, 12(s2)                                \n\t"
        ".if ATOM_M <= 5                                            \n\t"
        "add             s2, s2, %[A_BACK_STRIDE]                   \n\t"
        ".else                                                      \n\t"
        "add             s2, s2, %[A_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "flw             f20, 0(s2)                                 \n\t"
        "flw             f21, 4(s2)                                 \n\t"
        "flw             f22, 8(s2)                                 \n\t"
        "flw             f23, 12(s2)                                \n\t"
        ".if ATOM_M <= 6                                            \n\t"
        "add             s2, s2, %[A_BACK_STRIDE]                   \n\t"
        ".else                                                      \n\t"
        "add             s2, s2, %[A_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "flw             f24, 0(s2)                                 \n\t"
        "flw             f25, 4(s2)                                 \n\t"
        "flw             f26, 8(s2)                                 \n\t"
        "flw             f27, 12(s2)                                \n\t"
        ".if ATOM_M <= 7                                            \n\t"
        "add             s2, s2, %[A_BACK_STRIDE]                   \n\t"
        ".else                                                      \n\t"
        "add             s2, s2, %[A_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v0, (s3)                                   \n\t"
        ".if ATOM_N <= 4                                            \n\t"
        "add             s3, s3, 64                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v1, (s3)                                   \n\t"
        ".if ATOM_N <= 8                                            \n\t"
        "add             s3, s3, 48                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v2, (s3)                                   \n\t"
        ".if ATOM_N <= 12                                           \n\t"
        "add             s3, s3, 32                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v3, (s3)                                   \n\t"
        ".if ATOM_N <= 16                                           \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"

        ".if ATOM_N > 0                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v4, f0, v0                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v8, f4, v0                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v12, f8, v0                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v16, f12, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v20, f16, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v24, f20, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v28, f24, v0                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v5, f0, v1                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v9, f4, v1                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v13, f8, v1                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v17, f12, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v21, f16, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v25, f20, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v29, f24, v1                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v6, f0, v2                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v10, f4, v2                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v14, f8, v2                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v18, f12, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v22, f16, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v26, f20, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v30, f24, v2                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v7, f0, v3                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v11, f4, v3                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v15, f8, v3                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v19, f12, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v23, f16, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v27, f20, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v31, f24, v3                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"

        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v0, (s3)                                   \n\t"
        ".if ATOM_N <= 4                                            \n\t"
        "add             s3, s3, 64                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v1, (s3)                                   \n\t"
        ".if ATOM_N <= 8                                            \n\t"
        "add             s3, s3, 48                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v2, (s3)                                   \n\t"
        ".if ATOM_N <= 12                                           \n\t"
        "add             s3, s3, 32                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v3, (s3)                                   \n\t"
        ".if ATOM_N <= 16                                           \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"

        ".if ATOM_N > 0                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v4, f1, v0                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v8, f5, v0                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v12, f9, v0                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v16, f13, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v20, f17, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v24, f21, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v28, f25, v0                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v5, f1, v1                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v9, f5, v1                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v13, f9, v1                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v17, f13, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v21, f17, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v25, f21, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v29, f25, v1                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v6, f1, v2                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v10, f5, v2                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v14, f9, v2                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v18, f13, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v22, f17, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v26, f21, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v30, f25, v2                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v7, f1, v3                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v11, f5, v3                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v15, f9, v3                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v19, f13, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v23, f17, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v27, f21, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v31, f25, v3                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"

        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v0, (s3)                                   \n\t"
        ".if ATOM_N <= 4                                            \n\t"
        "add             s3, s3, 64                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v1, (s3)                                   \n\t"
        ".if ATOM_N <= 8                                            \n\t"
        "add             s3, s3, 48                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v2, (s3)                                   \n\t"
        ".if ATOM_N <= 12                                           \n\t"
        "add             s3, s3, 32                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v3, (s3)                                   \n\t"
        ".if ATOM_N <= 16                                           \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"

        ".if ATOM_N > 0                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v4, f2, v0                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v8, f6, v0                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v12, f10, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v16, f14, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v20, f18, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v24, f22, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v28, f26, v0                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v5, f2, v1                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v9, f6, v1                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v13, f10, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v17, f14, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v21, f18, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v25, f22, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v29, f26, v1                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v6, f2, v2                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v10, f6, v2                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v14, f10, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v18, f14, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v22, f18, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v26, f22, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v30, f26, v2                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v7, f2, v3                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v11, f6, v3                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v15, f10, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v19, f14, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v23, f18, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v27, f22, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v31, f26, v3                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"

        ".if ATOM_N > 0                                             \n\t"
        "vle.v           v0, (s3)                                   \n\t"
        ".if ATOM_N <= 4                                            \n\t"
        "add             s3, s3, 64                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vle.v           v1, (s3)                                   \n\t"
        ".if ATOM_N <= 8                                            \n\t"
        "add             s3, s3, 48                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vle.v           v2, (s3)                                   \n\t"
        ".if ATOM_N <= 12                                           \n\t"
        "add             s3, s3, 32                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vle.v           v3, (s3)                                   \n\t"
        ".if ATOM_N <= 16                                           \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".else                                                      \n\t"
        "add             s3, s3, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"

        ".if ATOM_N > 0                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v4, f3, v0                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v8, f7, v0                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v12, f11, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v16, f15, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v20, f19, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v24, f23, v0                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v28, f27, v0                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v5, f3, v1                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v9, f7, v1                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v13, f11, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v17, f15, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v21, f19, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v25, f23, v1                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v29, f27, v1                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v6, f3, v2                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v10, f7, v2                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v14, f11, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v18, f15, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v22, f19, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v26, f23, v2                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v30, f27, v2                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        ".if ATOM_M > 0                                             \n\t"
        "vfmacc.vf       v7, f3, v3                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        "vfmacc.vf       v11, f7, v3                                \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        "vfmacc.vf       v15, f11, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        "vfmacc.vf       v19, f15, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        "vfmacc.vf       v23, f19, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        "vfmacc.vf       v27, f23, v3                               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        "vfmacc.vf       v31, f27, v3                               \n\t"
        ".endif                                                     \n\t"
        ".endif                                                     \n\t"

        "bnez            s5, 1b                                     \n\t"

    "2:                                                             \n\t" // store C
        ".if ATOM_M > 0                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vse.v           v4, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vse.v           v5, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vse.v           v6, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vse.v           v7, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 1                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vse.v           v8, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vse.v           v9, (s4)                                   \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vse.v           v10, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vse.v           v11, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 2                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vse.v           v12, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vse.v           v13, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vse.v           v14, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vse.v           v15, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 3                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vse.v           v16, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vse.v           v17, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vse.v           v18, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vse.v           v19, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 4                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vse.v           v20, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vse.v           v21, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vse.v           v22, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vse.v           v23, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 5                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vse.v           v24, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vse.v           v25, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vse.v           v26, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vse.v           v27, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_M > 6                                             \n\t"
        ".if ATOM_N > 0                                             \n\t"
        "vse.v           v28, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 4                                             \n\t"
        "vse.v           v29, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 8                                             \n\t"
        "vse.v           v30, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        ".if ATOM_N > 12                                            \n\t"
        "vse.v           v31, (s4)                                  \n\t"
        "add             s4, s4, 16                                 \n\t"
        ".endif                                                     \n\t"
        "add             s4, s4, %[C_NXT_LINE_STRIDE]               \n\t"
        ".endif                                                     \n\t"

        :
        :
        [ATOM_M]"i"(m),
        [ATOM_N]"i"(n),
        [IS_FIRST]"i"(first),
        [A_LOC]"r"(a),
        [B_LOC]"r"(b),
        [C_LOC]"r"(c),
        [BIAS_LOC]"r"(bias),
        [K]"r"(total_k),
        [A_NXT_LINE_STRIDE]"r"(total_k * 4),
        [A_BACK_STRIDE]"r"(-((m - 1) * total_k * 4 - 16)),
        [C_NXT_LINE_STRIDE]"r"((total_n - n) * 4)

        : "memory", "s2", "s3", "s4", "s5", "s6"
    );
}

template<int64_t align_m, int64_t align_n, int64_t left_m, int64_t left_n, bool first>
void fc_ndarray_gemm_kernel_fp32_vec128(const float *a, const float *b, const float* bias, float *c, int32_t total_m, int32_t total_n, int32_t total_k) {
    int64_t mi = 0;
    for (mi = 0; mi <= total_m - align_m; mi += align_m) {
        int64_t ni = 0;
        auto temp_a = a + mi * total_k;
        for (ni = 0; ni <= total_n - align_n; ni += align_n) {
            auto temp_b = b + ni * total_k;
            auto temp_c = c + mi * total_n + ni;
            auto temp_bias = bias + ni;
            fc_ndarray_gemm_kernel_m7n16_fp32_vec128<align_m, align_n, first>(temp_a, temp_b, temp_bias, temp_c, total_m, total_n, total_k);
        }
        if (ni < total_n) {
            auto temp_b = b + ni * total_k;
            auto temp_c = c + mi * total_n + ni;
            auto temp_bias = bias + ni;
            fc_ndarray_gemm_kernel_m7n16_fp32_vec128<align_m, left_n, first>(temp_a, temp_b, temp_bias, temp_c, total_m, total_n, total_k);
        }
    }

    if (mi < total_m) {
        int64_t ni = 0;
        auto temp_a = a + mi * total_k;
        for (ni = 0; ni <= total_n - align_n; ni += align_n) {
            auto temp_b = b + ni * total_k;
            auto temp_c = c + mi * total_n + ni;
            auto temp_bias = bias + ni;
            fc_ndarray_gemm_kernel_m7n16_fp32_vec128<left_m, align_n, first>(temp_a, temp_b, temp_bias, temp_c, total_m, total_n, total_k);
        }
        if (ni < total_n) {
            auto temp_b = b + ni * total_k;
            auto temp_c = c + mi * total_n + ni;
            auto temp_bias = bias + ni;
            fc_ndarray_gemm_kernel_m7n16_fp32_vec128<left_m, left_n, first>(temp_a, temp_b, temp_bias, temp_c, total_m, total_n, total_k);
        }
    }
}

template<bool first>
fc_common_gemm_kernel_func_t<float> fc_ndarray_select_gemm_kernel_fp32_vec128(int32_t m, int32_t n) {
    switch (n % 16) {
        case 4: switch (m % 7) {
            case 0: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 7, 4, first>;
            case 1: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 1, 4, first>;
            case 2: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 2, 4, first>;
            case 3: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 3, 4, first>;
            case 4: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 4, 4, first>;
            case 5: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 5, 4, first>; 
            case 6: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 6, 4, first>;
        }
        case 8: switch (m % 7) {
            case 0: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 7, 8, first>;
            case 1: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 1, 8, first>;
            case 2: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 2, 8, first>;
            case 3: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 3, 8, first>;
            case 4: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 4, 8, first>;
            case 5: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 5, 8, first>;
            case 6: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 6, 8, first>;
        }
        case 12: switch (m % 7) {
            case 0: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 7, 12, first>;
            case 1: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 1, 12, first>;
            case 2: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 2, 12, first>;
            case 3: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 3, 12, first>;
            case 4: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 4, 12, first>;
            case 5: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 5, 12, first>;
            case 6: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 6, 12, first>;
        }    
        case 0: switch (m % 7) {
            case 0: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 7, 16, first>;
            case 1: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 1, 16, first>;
            case 2: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 2, 16, first>;
            case 3: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 3, 16, first>;
            case 4: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 4, 16, first>;
            case 5: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 5, 16, first>;
            case 6: return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 6, 16, first>;
        }
    }
    return fc_ndarray_gemm_kernel_fp32_vec128<7, 16, 7, 16, first>;
}

}}}; // namespace ppl::kernel::riscv

#endif